{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 2.1.0\n",
      "\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "loading embedding ...\n",
      "embedding loaded as `glove_embedding`\n"
     ]
    }
   ],
   "source": [
    "%run init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = mz.preprocessors.BasicPreprocessor(fixed_length_left=10, fixed_length_right=40, remove_stop_words=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 5660.56it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4467.85it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 786301.58it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 122848.89it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 139092.07it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 454679.90it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 669810.24it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 404432/404432 [00:00<00:00, 2422809.59it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 8241.89it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4600.64it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 118459.97it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 188154.70it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 41857.09it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 588586.49it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 725717.97it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 114181.33it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 89142.16it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 122/122 [00:00<00:00, 8251.31it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 1115/1115 [00:00<00:00, 4700.14it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 119923.30it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 97728.24it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 124012.86it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 181842.60it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 544176.05it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 89147.23it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 89651.09it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 237/237 [00:00<00:00, 8348.80it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2300/2300 [00:00<00:00, 4593.53it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 131653.35it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 165840.85it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 132096.83it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 273842.99it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 639163.80it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 89064.60it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 86151.49it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "dev_pack_processed = preprocessor.transform(dev_pack_raw)\n",
    "test_pack_processed = preprocessor.transform(test_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x7f30f43876a0>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x7f3176e6aeb8>,\n",
       " 'vocab_size': 16674,\n",
       " 'embedding_input_dim': 16674,\n",
       " 'input_shapes': [(10,), (40,)]}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_class                   <class 'matchzoo.contrib.models.match_lstm.MatchLSTM'>\n",
      "input_shapes                  [(10,), (40,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adadelta\n",
      "with_embedding                True\n",
      "embedding_input_dim           16674\n",
      "embedding_output_dim          100\n",
      "embedding_trainable           True\n",
      "lstm_num_units                100\n",
      "fc_num_units                  100\n",
      "dropout_rate                  0.5\n"
     ]
    }
   ],
   "source": [
    "model = mz.contrib.models.MatchLSTM()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params['task'] = ranking_task\n",
    "model.params['embedding_output_dim'] = 100\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['fc_num_units'] = 100\n",
    "model.params['lstm_num_units'] = 100\n",
    "model.params['dropout_rate'] = 0.5\n",
    "model.params['optimizer'] = 'adadelta'\n",
    "model.guess_and_fill_missing_params()\n",
    "model.build()\n",
    "model.compile()\n",
    "print(model.params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 40)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           multiple             1667400     text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lstm_left (LSTM)                (None, 10, 100)      80400       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_right (LSTM)               (None, 40, 100)      80400       embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, 10, 100)      0           lstm_left[0][0]                  \n",
      "                                                                 lstm_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 50, 100)      0           lambda_10[0][0]                  \n",
      "                                                                 lstm_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lstm_merge (LSTM)               (None, 200)          240800      concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 200)          0           lstm_merge[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, 100)          20100       dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_24 (Dense)                (None, 1)            101         dense_23[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 2,089,201\n",
      "Trainable params: 2,089,201\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.backend.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = glove_embedding.build_matrix(preprocessor.context['vocab_unit'].state['term_index'])\n",
    "model.load_embedding_matrix(embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x, test_y = test_pack_processed.unpack()\n",
    "evaluate = mz.callbacks.EvaluateAllMetrics(model, x=test_x, y=test_y, batch_size=len(test_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num batches: 102\n"
     ]
    }
   ],
   "source": [
    "train_generator = mz.DataGenerator(\n",
    "    train_pack_processed,\n",
    "    mode='pair',\n",
    "    num_dup=2,\n",
    "    num_neg=1,\n",
    "    batch_size=20\n",
    ")\n",
    "print('num batches:', len(train_generator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "102/102 [==============================] - 13s 130ms/step - loss: 0.9022\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.56891574533972 - normalized_discounted_cumulative_gain@5(0.0): 0.6259075966908896 - mean_average_precision(0.0): 0.5895521163084454\n",
      "Epoch 2/10\n",
      "102/102 [==============================] - 11s 106ms/step - loss: 0.6955\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5876851995953162 - normalized_discounted_cumulative_gain@5(0.0): 0.6407140437458756 - mean_average_precision(0.0): 0.5965985760516177\n",
      "Epoch 3/10\n",
      "102/102 [==============================] - 11s 105ms/step - loss: 0.6073\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6151453530205596 - normalized_discounted_cumulative_gain@5(0.0): 0.6639169915844698 - mean_average_precision(0.0): 0.6198851976278136\n",
      "Epoch 4/10\n",
      "102/102 [==============================] - 11s 105ms/step - loss: 0.5805\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6097028553948147 - normalized_discounted_cumulative_gain@5(0.0): 0.6654420380026644 - mean_average_precision(0.0): 0.6240460033736575\n",
      "Epoch 5/10\n",
      "102/102 [==============================] - 11s 108ms/step - loss: 0.5180\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6133176603089614 - normalized_discounted_cumulative_gain@5(0.0): 0.6538666262678027 - mean_average_precision(0.0): 0.6188266626371615\n",
      "Epoch 6/10\n",
      "102/102 [==============================] - 11s 107ms/step - loss: 0.4860\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6196634602764765 - normalized_discounted_cumulative_gain@5(0.0): 0.6765955662781967 - mean_average_precision(0.0): 0.6318559407749947\n",
      "Epoch 7/10\n",
      "102/102 [==============================] - 11s 107ms/step - loss: 0.4297\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5911951516660675 - normalized_discounted_cumulative_gain@5(0.0): 0.6356282828122544 - mean_average_precision(0.0): 0.5974699334878332\n",
      "Epoch 8/10\n",
      "102/102 [==============================] - 11s 105ms/step - loss: 0.3946\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6316524262499843 - normalized_discounted_cumulative_gain@5(0.0): 0.6774169076547488 - mean_average_precision(0.0): 0.6368686569077484\n",
      "Epoch 9/10\n",
      "102/102 [==============================] - 11s 106ms/step - loss: 0.3788\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5969334279811508 - normalized_discounted_cumulative_gain@5(0.0): 0.6513736764474628 - mean_average_precision(0.0): 0.607785113937385\n",
      "Epoch 10/10\n",
      "102/102 [==============================] - 11s 106ms/step - loss: 0.3170\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6266464172038838 - normalized_discounted_cumulative_gain@5(0.0): 0.6722764129615637 - mean_average_precision(0.0): 0.6201359089808456\n"
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(train_generator, epochs=10, callbacks=[evaluate], workers=4, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use this function to update the README.md with a better set of parameters. Make sure you delete the correct section of the README.md before calling this function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "append_params_to_readme(model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
